<?php
namespace app\api\controller;


use XCrawler\XCrawler;
use GuzzleHttp\Client;
use Symfony\Component\DomCrawler\Crawler;
use think\Db;
class Wcrawler{

    private $_key = '';

    private $_public_name = '';

    public function __construct($name,$key){
        $this->_public_name = $name;

        $this->_key = $key;
    }


    private function _getCookieFile(){
        $cookie_arr = redis()->hgetall($this->_key.'_cookie');
        $cookie='';
        foreach ($cookie_arr as $k=>$v){
            $cookie .= $k.'='.$v.';';
        }
        return  $cookie;
    }


    private function _getToken(){
        return redis()->get($this->_key.'_token');
    }


    public function index(){
        require_once 'vendor/autoload.php';


        $config = [
            'name' => 'public',
//            'concurrency' => 3,
            'timeout' => 10.0,
            'interval' => 3,
            'retry_count' => 3,
            'requests' => function() {
                $requests = [
                    'method' => 'get',
                    'uri' => 'https://mp.weixin.qq.com/cgi-bin/searchbiz',
                    'verify'=>false,
//                    'callback_data' => [
//                        'is_first_page' => 1,
//                    ],
                    'headers'=>[
                        'Accept' => 'application/json, text/javascript, */*; q=0.01',
                        'Accept-Encoding' => 'gzip, deflate, br',
                        'Accept-Language' => 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
                        'Cache-Control' => 'max-age=0',
                        'Host' => 'mp.weixin.qq.com',
                        'User-Agent' => 'Mozilla/5.0 (Linux; U; Android 4.4.3; zh-cn; HTC 802w Build/KTU84L) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025489 Mobile Safari/533.1 MicroMessenger/6.3.13.49_r4080b63.740 NetType/WIFI Language/zh_CN',
                        'Connection' => 'keep-alive',
                        'Referer'=>'https://mp.weixin.qq.com/cgi-bin/appmsg?t=media/appmsg_edit_v2&action=edit&isNew=1&type=10&lang=zh_CN&token='.$this->_getToken(),
                        'Cookie'=>$this->_getCookieFile(),
                    ],

                    'query' => [
                        'action' =>	'search_biz',
                        'ajax' => 1,
                        'begin' => 0,
                        'count' => 5,
                        'f' => 'json',
                        'lang' => 'zh_CN',
                        'query' => $this->_public_name,
                        'random' => '0.'.mt_rand(100000000, 999999999),
                        'token' => $this->_getToken(),
                    ],
                ];
                yield $requests;
            },

            'success' => function($result, $request, $xcrawler, $res_headers) {
                $res = json_decode($result,true);
                if(isset($res['list'])){
                    redis()->lpush('wxpublic:public', json_encode($res['list'][0]));
                }else{
                    echo '帐号登录已过期！！';
                    return false;
                }

            }
        ];


        $xcrawler = new XCrawler($config);

        $xcrawler->run();


        $this->content_list();
    }





    private function content_list(){
        require_once 'vendor/autoload.php';

        $config = [
            'name' => 'public:article_list',
            'concurrency' => 3,
            'timeout' => 10.0,
            'interval' => 3,
            'retry_count' => 3,

            'requests' => function() {
                while ($data = redis()->rpop('wxpublic:public')) {
                    $data = json_decode($data, true);
                    $requests = [
                        'method' => 'get',
                        'uri' => 'https://mp.weixin.qq.com/cgi-bin/appmsg',
                        'verify'=>false,
                        'callback_data' => [
                            'is_first_page' => 1,
                            'request_data' => $data,
                        ],
                        'headers'=>[
                            'Accept' => 'application/json, text/javascript, */*; q=0.01',
                            'Accept-Encoding' => 'gzip, deflate, br',
                            'Accept-Language' => 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
                            'Cache-Control' => 'max-age=0',
                            'Host' => 'mp.weixin.qq.com',
                            'User-Agent' => 'Mozilla/5.0 (Linux; U; Android 4.4.3; zh-cn; HTC 802w Build/KTU84L) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025489 Mobile Safari/533.1 MicroMessenger/6.3.13.49_r4080b63.740 NetType/WIFI Language/zh_CN',
                            'Connection' => 'keep-alive',
                            'Referer'=>'https://mp.weixin.qq.com/cgi-bin/appmsg?t=media/appmsg_edit_v2&action=edit&isNew=1&type=10&lang=zh_CN&token='.$this->_getToken(),
                            'Cookie'=>$this->_getCookieFile(),
                        ],

                        'query' => [
                            'action' =>	'list_ex',
                            'ajax' => 1,
                            'begin' => 0,
                            'count' => 5,
                            'f' => 'json',
                            'lang' => 'zh_CN',
                            'query' => '',
                            'random' => '0.'.mt_rand(100000000, 999999999),
                            'token' => $this->_getToken(),
                            'fakeid' => $data['fakeid'],
                            'type' => 9,
                        ],
                    ];
                    yield $requests;
                }
            },

            'success' => function($result, $request, $xcrawler, $res_headers) {
                $res = json_decode($result,true);

                if($res['base_resp']['ret'] == 0){
                    foreach ($res['app_msg_list'] as $v){
                        redis()->lpush('wxpublic:public_article_list', json_encode($v));
                    }
                }else{
//                    redis()->lpush('wxpublic:public', json_encode($request['data']));
                    redis()->set('wxpublic:public_article_page',$request['query']['begin']);
                    echo "爬取错误结束\n";
                    return false;
                }

                // 判断是否是第一页
                if ($request['callback_data']['is_first_page'] == 1 && $res['app_msg_cnt']>5) {

                    $fakeid = $request['callback_data']['request_data']['fakeid'];

                    $page = redis()->get('wxpublic:public_article_page');

                    for ($i = $page>0?$page:5;$i <= $res['app_msg_cnt']; ) {
                        $requests = [
                            'method' => 'get',
                            'uri' => 'https://mp.weixin.qq.com/cgi-bin/appmsg',
                            'verify'=>false,
                            'callback_data' => [
                                'is_first_page' => 0
                            ],
                            'headers'=>[
                                'Accept' => 'application/json, text/javascript, */*; q=0.01',
                                'Accept-Encoding' => 'gzip, deflate, br',
                                'Accept-Language' => 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
                                'Cache-Control' => 'max-age=0',
                                'Host' => 'mp.weixin.qq.com',
                                'User-Agent' => 'Mozilla/5.0 (Linux; U; Android 4.4.3; zh-cn; HTC 802w Build/KTU84L) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025489 Mobile Safari/533.1 MicroMessenger/6.3.13.49_r4080b63.740 NetType/WIFI Language/zh_CN',
                                'Connection' => 'keep-alive',
                                'Referer'=>'https://mp.weixin.qq.com/cgi-bin/appmsg?t=media/appmsg_edit_v2&action=edit&isNew=1&type=10&lang=zh_CN&token='.$this->_getToken(),
                                'Cookie'=>$this->_getCookieFile(),
                            ],

                            'query' => [
                                'action' =>	'list_ex',
                                'ajax' => 1,
                                'begin' => $i,
                                'count' => 5,
                                'f' => 'json',
                                'lang' => 'zh_CN',
                                'query' => '',
                                'random' =>'0.'.mt_rand(100000000, 999999999),
                                'token' => $this->_getToken(),
                                'fakeid' => $fakeid,
                                'type' => 9,
                            ],
                        ];

                        $i = $i+5;
                        // 添加下一页的请求到请求队列
                        $xcrawler->addRequest($requests);
                    }
                }


            }
        ];


        $xcrawler = new XCrawler($config);
        $xcrawler->run();


        $this->get_content();
    }





    private function get_content(){
        require_once 'vendor/autoload.php';


        $config = [
            'name' => 'public:article_content',
            'concurrency' => 6,
            'timeout' => 10.0,
            'interval' => 1,
            'retry_count' => 3,

            'requests' => function() {
                while ($data = redis()->rpop('wxpublic:public_article_list')) {
                    $data = json_decode($data, true);
                    $requests = [
                        'method' => 'get',
                        'uri' => $data['link'],
                        'verify'=>false,
                        'callback_data' => $data,
//                        'callback_data' => [
//                            'is_first_page' => 1,
//                            'request_data' => $data,
//                        ],
                        'headers'=>[
                            'Accept' => 'text/html,application/xhtml+xm…plication/xml;q=0.9,*/*;q=0.8',
                            'Accept-Encoding' => 'gzip, deflate, br',
                            'Accept-Language' => 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
                            'Cache-Control' => 'max-age=0',
                            'Host' => 'mp.weixin.qq.com',
                            'User-Agent' => 'Mozilla/5.0 (Linux; U; Android 4.4.3; zh-cn; HTC 802w Build/KTU84L) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025489 Mobile Safari/533.1 MicroMessenger/6.3.13.49_r4080b63.740 NetType/WIFI Language/zh_CN',
                            'Connection' => 'keep-alive',
                            'Cookie'=>$this->_getCookieFile(),
                        ],
                    ];
                    yield $requests;
                }
            },

            'success' => function($result, $request, $xcrawler, $res_headers) {

                redis_del('wxpublic:public_article_page');
                $crawler = new Crawler();
                $crawler->addHtmlContent($result);

//                 $content = $crawler->filter('#js_content')->html();

                $content = dom_filter($crawler, '#js_content', 'html');



                if($content){
                    $content = trim(strtr($content, ['data-src' => 'src','opacity: 0;'=>'']));

                    $data = $request['callback_data'];
                    $img_url = $this->img_download('',$data);
                    $data['cover'] = $img_url;
                    $data['content'] = $content;

//                     $seletor = '#js_content > p';
                    $seletor = '//*[@id="js_content"]/p';

                    $crawler->filterXPath($seletor)->each(function (Crawler $node, $i) use (&$data) {

                        $img = dom_filter($node, 'img', 'attr','data-src');

                        if(!empty($img)){
                            $after_img_url = $this->img_download($img,$data);
                            $data['content'] = strtr($data['content'],[$img=>$after_img_url]);
                        }

                    });


                    echo $data['title']."\n";

                    $savedata=[];
                    $savedata['wx_title']=$data['title'];
                    $savedata['wx_content']=$data['content'];
                    $savedata['wx_pulish_time']=$data['update_time'];
                    $savedata['wx_intro']=$data['digest'];
                    $savedata['wx_cover']=$data['cover'];
                    $savedata['wx_aid']=$data['aid'];
                    $savedata['wx_link']=$data['link'];

                    Db::name('wx_public')->insert($savedata);
                }

//                 echo $content;
//                // 通过css选择器遍历影片列表
//                $seletor = '  > dl a';
//                $data = $request['callback_data'];
//                $crawler->filter($seletor)->each(function (Crawler $node, $i) use (&$data) {
//                    $data['chapte_url'] = $node->attr('href');
//                    $data['chapte_name'] = $node->html();
//                    // 把影片url、name推送到redis队列，以便进一步爬取影片下载链接
//                    redis()->lpush('biquge:chapte_queue', json_encode($data));
//                });

//                if(isset($res['list'])){
//                    redis()->lpush('wxpublic:public', json_encode($res['list'][0]));
//                }

//                $list = [];
//                // 通过css选择器遍历影片列表
//                $tr_selector = '#hotcontent > table tr';
//                $crawler->filter($tr_selector)->each(function (Crawler $node, $i) use (&$list) {
//                    $name = dom_filter($node, 'td:nth-child(1) > a', 'html');
//                    if (empty($name)) {
//                        return;
//                    }
//                    $url = dom_filter($node, 'td:nth-child(1) > a', 'attr', 'href');
//
//                    $data = [
//                        'titlename' => $name,
//                        'list_url' => $url,
//                        'crawler_time' => time(),
//                    ];
//                    // 把影片url、name推送到redis队列，以便进一步爬取影片下载链接

//                    $list[] = $data;
////                    var_dump($data);
//                });
//
            },


            'error' => function($request,$error_msg){
                $data = $request['callback_data'];
                redis()->lpush('wxpublic:public_article_list', json_encode($data));
                var_dump($error_msg);
            }
        ];


        $xcrawler = new XCrawler($config);
        $xcrawler->run();
    }



    private function img_download($img_url = '',$data){
        $headers = [
            'headers' => [
                'Accept' => 'text/html,application/xhtml+xm…plication/xml;q=0.9,*/*;q=0.8',
                'Accept-Encoding' => 'gzip, deflate, br',
                'Accept-Language' => 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
                'Cache-Control' => 'max-age=0',
                'Host' => 'mmbiz.qpic.cn',
                'User-Agent' => 'Mozilla/5.0 (Linux; U; Android 4.4.3; zh-cn; HTC 802w Build/KTU84L) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025489 Mobile Safari/533.1 MicroMessenger/6.3.13.49_r4080b63.740 NetType/WIFI Language/zh_CN',
                'Connection' => 'keep-alive'
            ],
            'verify'=>false
        ];


        $client = new Client();

        $response= $client->request('GET',$img_url?$img_url:$data['cover'],$headers);



        if($response->getStatusCode() == 200){
            $res = $response->getBody()->getContents();


            $path = '/wxpublic_article/'.$data['aid'];
            !is_dir('./public'.$path) && mkdir('./public'.$path, 0755, true);

            if($img_url){
                preg_match('/wx_fmt=(jpeg|png|jpg|gif)/', $img_url,$matches);
                if(empty($matches[1])){
                    $matches[1]='jpg';
                }

                $filename = $path.'/'.uniqid().'.'.$matches[1];
            }else{
                preg_match('/wx_fmt=(jpeg|png|jpg|gif)/', $data['cover'],$matches);
                if(empty($matches[1])){
                    $matches[1]='jpg';
                }

                $filename = $path.'/cover.'.$matches[1];
            }



            $fp         = fopen('./public'.$filename, "wb+") or die("open fails");
            fwrite($fp,$res) or die("fwrite fails");
            fclose($fp);
            return $filename;
        }


    }

}